package stardust.product.spider.http;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.Date;
import java.util.List;
import stardust.core.network.protocol.http.HttpClient;
import stardust.core.network.protocol.http.HttpMethod;
import stardust.core.network.protocol.http.HttpResponseMeta;
import stardust.core.network.protocol.http.support.URL;
import stardust.core.support.db.DataAccessObject;
import stardust.core.support.db.StardustDB;
import stardust.product.spider.FetchHTML;
import stardust.product.spider.FetchURL;
import stardust.product.spider.WebcSpider;
import stardust.product.spider.URLSource.URLEntity;

public class HTTPWebcSpider extends WebcSpider
{
  
  public static final String CLIENT = "Stardust Spider 1.0";

  public void runTask(URLEntity urlEntity)
  {
    HttpResponseMeta response;
    HttpClient httpClient;
    
    httpClient = new HttpClient(HTTPWebcSpider.CLIENT);
    
    try
    {
      if (urlEntity.getReferer() != null)
      {
        httpClient.setReferer(urlEntity.refererToURL());
  //      httpClient.setReferer(null);A
      }
      else
      {
        httpClient.setReferer(null);
      }
      
      response = httpClient.go(urlEntity.getUrl(), HttpMethod.get);
      super.fetchCount++;
      this.detect(urlEntity.getUrl(), response);
    }
    catch (Exception e)
    {
      e.printStackTrace();
    }
    finally
    {
      httpClient.disconnect();
    }
  }
  
  public void detect(String url, HttpResponseMeta response)
  {
    // 处理爬虫抓取页面失败。
    DataAccessObject dao = null;
    
    try
    {
      dao = StardustDB.getDataAccessObject();
      
      if (response == null)
      {
        dao.execute("UPDATE TB_MOD_SPD_PAGE SET SP_UPDATE_TIME = NOW(), SP_FLAG = 3 WHERE SP_URL = ?",  url);
      }
      else
      {
        dao.execute("UPDATE TB_MOD_SPD_PAGE SET SP_UPDATE_TIME = NOW(), SP_FLAG = 2, SP_HEADER = ?, SP_STATUS_CODE = ?, SP_ERROR_MESSAGE = ? WHERE SP_URL = ?", response.getOthers().get("Content-Type"), response.getStatusCode(), response.getDescription(),  url);
      }
      dao.execute("COMMIT");
    }
    finally
    {
      if (dao != null)
      {
        dao.close(true);
        dao = null;
      }
    }
    
    if (response == null)
    {
      return;
    }
    
    // 精确处理有Content-Type头的内容，其它内容只做保存。
    if (response.getOthers().get("Content-Type") != null)
    {
      String contentType = response.getOthers().get("Content-Type").trim();
      
      if (contentType.equalsIgnoreCase("text/html"))
      {
        // 处理HTML文件。
        String pageEncoding;
        String html;
        
        if (contentType.indexOf('=') != -1)
        {
          pageEncoding = contentType.substring(contentType.indexOf('='));
        }
        else
        {
          pageEncoding = "gb2312";
        }
        
        try
        {
          html = new String(response.getData(), pageEncoding);
        }
        catch (UnsupportedEncodingException e)
        {
          e.printStackTrace();
          return;
        }
        
        // 检测游民星空下载页面的关键要素。
        if (html.indexOf("下载地址") != -1 && html.indexOf("下载说明") != -1 && html.indexOf("下载属性") != -1)
        {
          try
          {
            System.err.println(super.fetchCount + " - " + new Date().toLocaleString() + "\t收录[" + response.getStatusCode() + "] -> " + url);
            
            dao = StardustDB.getDataAccessObject();
            
            if (dao.get("SELECT * FROM TB_MOD_WEB_RESOURCE WHERE WR_URL = ?", url) == null)
            {
              dao.execute("INSERT INTO TB_MOD_WEB_RESOURCE VALUES (NULL, NOW(), '游戏', 5, '游民星空', 0, 0, ?, NULL, ?, ?)", url, FetchHTML.fetchTitle(html), FetchHTML.fetchTitle(html));
            }
          }
          finally
          {
            if (dao != null)
            {
              dao.close(false);
            }
          }
        }
        else
        {
          System.out.println(super.fetchCount + " - " + new Date().toLocaleString() + "\t丢弃[" + response.getStatusCode() + "] -> " + url);
        }
        
        // 最后提取页面中的URL信息（A标签）。
        this.fetchUrl(url, html);
      }
      else if (contentType.equalsIgnoreCase("application/x-bittorrent"))
      {
        // 检测游民星空的BT文件。
        try
        {
          System.err.println(super.fetchCount + " - " + new Date().toLocaleString() + "\t收录BT文件[" + response.getStatusCode() + "] -> " + url);
          
          dao = StardustDB.getDataAccessObject();
          
          dao.execute("INSERT INTO TB_MOD_WEB_RESOURCE_ADDRESS SELECT NULL, NOW(), (SELECT WR_ID FROM TB_MOD_WEB_RESOURCE WHERE WR_URL = (SELECT SP_REFERER FROM TB_MOD_SPD_PAGE WHERE SP_URL = ?)), 'BT', NULL, (SELECT SP_REFERER FROM TB_MOD_SPD_PAGE WHERE SP_URL = ?), ?", url, url, url);
        }
        finally
        {
          if (dao != null)
          {
            dao.close(false);
          }
        }
      }
    }
    else
    {
      System.out.println(super.fetchCount + " - " + new Date().toLocaleString() + "\t丢弃[" + response.getStatusCode() + "] -> " + url);
    }
  }
  
  public void fetchUrl(String referer, String html)
  {
    List<String> foundUrl = FetchURL.fetchLink(referer, html);
    
    DataAccessObject dao = null;
    
    try
    {
      dao = StardustDB.getDataAccessObject();
      
      for (int i = 0; i < foundUrl.size(); i++)
      {
        if (dao.get("SELECT SP_URL FROM TB_MOD_SPD_PAGE WHERE SP_URL = ? LIMIT 0, 1", foundUrl.get(i)) != null)
        {
          continue;
        }
        else
        {
          URL u;
          
          try
          {
            u = new URL(foundUrl.get(i));
          }
          catch (Exception e)
          {
            System.out.println("ERROR PARSE URL -> " + foundUrl.get(i));
            continue;
          }
          
          // 如果存在对应的处理程序将写入数据库，否则将丢弃。
          if (u.getHost().equalsIgnoreCase("www.gamersky.com") || u.getHost().equalsIgnoreCase("db.gamersky.com"))
          {
            dao.execute
            (
              "INSERT INTO TB_MOD_SPD_PAGE VALUES (NULL, NOW(), NULL, 0, NULL, NULL, NULL, ?, ?, ?, ?)",
              u.getHost(), u.getPort() == -1 ? 80 : u.getPort(), referer, foundUrl.get(i)
            );
          }
          dao.execute("COMMIT");
        }
      }
    }
    finally
    {
      if (dao != null)
      {
        dao.close(true);
      }
    }
  }

}
